%%html
<img src="Bilder Notebook/Output_Img_Colored0.jpg", width=500>
In this chapter we are focused on finding the lane lines in an image and later on test it on a video. This is now a more deeper way to find the lanes in the image. Short description of the way how I fixed it:
First we start with importing all important modules and the load the calibration values.
import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import math
import function as fnc
import FindingLane as lane
#Read Image
def readIMG(img_name):
image = mpimg.imread(img_name)
shape = image.shape
color_img = np.copy(image)
return color_img, shape
# read the calibration values ¶
dist_pickle = pickle.load( open("Camera_Calib_Values/wide_dist_pickle9Corner_class.p", "rb" ) )
mtx = dist_pickle["mtx"]
dist = dist_pickle["dist"]
# get test-image
color_img, shape = fnc.readIMG('test_images/straight_lines1.jpg')
line = lane.Lines()
line.set_cam_calib_param(mtx,dist)
undist_Img = line.undistort(color_img)
#plot Images
images=[color_img, undist_Img]
titels=['Original', 'Undistored IMG']
fnc.plot_n(images,titels,'gray')
# Canny Edge Detection
low_threshold_canny = 50
high_threshold_canny = 150
sobel_kernel = 5
canny_binary = line.canny_edge(undist_Img,low_threshold_canny,high_threshold_canny,sobel_kernel, True)
Canny Edge Detection is not the best option. Therefore we try
Here we analyze each direction separate. The sobel operator make a convolution in X and Y direction.
%%html
<img src="Bilder Notebook/grad.png", width=300>
<img src="Bilder Notebook/SobelBeisp.png", width=400>
%%html
<img src="Bilder Notebook/SobelBeispLösung.png", width=200>
abs_sobelx=$ \sqrt{(sobel_x)^2}$
abs_sobely=$\sqrt{(sobel_y)^2}$
abs_sobelxy = $\sqrt{(sobel_x)^2+(sobel_y)^2}$
$arctan(sobely/sobelx)$
ksize = 9
gradx = line.abs_sobel_thresh(undist_Img, orient='x', sobel_kernel=ksize, thresh=(30,220))
grady = line.abs_sobel_thresh(undist_Img, orient='y', sobel_kernel=ksize, thresh=(30, 220))
mag_binary = line.mag_thresh(undist_Img, sobel_kernel=ksize, mag_thresh=(30, 220))
dir_binary = line.dir_threshold(undist_Img, sobel_kernel=ksize, thresh=(0.6, 1.1))
combined = np.zeros_like(dir_binary)
combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
images=[gradx, grady,mag_binary,dir_binary,combined]
titels=['Sobel X thresh', 'Sobel Y thresh','mag_binary','dir_binary','combined']
fnc.plot_n(images,titels,'gray')
A color space is a specific organization of colors; color spaces provide a way to categorize colors and represent them in digital images.
RGB is red-green-blue color space. You can think of this as a 3D space, in this case a cube, where any color can be represented by a 3D coordinate of R, G, and B values. For example, white has the coordinate (255, 255, 255), which has the maximum value for red, green, and blue.
%%html
<img src="Bilder Notebook/RGB.png", width=300>
There is also HSV color space (hue, saturation, and value), and HLS space (hue, lightness, and saturation). These are some of the most commonly used color spaces in image analysis. Here we use the HLS space
%%html
<img src="Bilder Notebook/HLS.png", width=300>
# -------------- Color Spaces ---------------------
#Binary Threshold
bin_thresh = (180, 255)
binary_img = line.binary_thresh(undist_Img, bin_thresh,True)
#RGB Thresh
rgb_thresh = (200, 255)
R_binary = line.rgb_thresh(undist_Img,rgb_thresh,True)
#HLS Thresh
thresh_hLs = (90, 255)
thresh_Hls = (15, 100)
hls_thresh = thresh_hLs + thresh_Hls
hls_s_binary, hls_h_binary = line.hls_thresh(undist_Img, hls_thresh,True)
combined = np.zeros_like(dir_binary)
combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
combined_Color = np.zeros_like(R_binary)
combined_Color[((hls_s_binary == 1) & (hls_h_binary == 1)) | (R_binary == 1)] = 1
combined_beide = np.zeros_like(combined_Color)
combined_beide[((combined_Color == 1) | (combined == 1))] = 1
images=[binary_img,R_binary,hls_s_binary, hls_h_binary, combined_Color,combined_beide ]
titels=['Binary Thresh','RGB Thresh','HLS -S- Thresh','HLS -H- Thresh', 'combined_Color','combined_both']
fnc.plot_n(images,titels,'gray')
This is a pipline with the outcome of the best combination result.
##----------------------Sobel Threshold
sobel_thresh = (30,220)
mag_thresh = (30, 220)
dir_thresh = (0.6, 1.1)
bin_thresh = (180, 255)
rgb_thresh = (200, 255)
hls_L_thresh = (90, 255)
hls_H_thresh = (15, 100)
hls_thresh = hls_L_thresh + hls_H_thresh
thresh = [sobel_thresh, mag_thresh, dir_thresh, bin_thresh, \
rgb_thresh, hls_thresh]
ksize = 9
binary_best_combi = line.pipline_combination(undist_Img, ksize, thresh, False)
images=[undist_Img,binary_best_combi]
titels=['Original IMG','Binary Best Combination']
fnc.plot_n(images,titels,'gray')
Define a Region of Interest (Polygon) like in the folloing image. The y_line is how far away the view should be. The function for that is in function.py included.
%%html
<img src="Bilder Notebook/ROI_marked.png", width=400>
# ROI - Four Side Polygone -> polygon_roi(img, imshape, y_line, x_left, x_right, rgb_threshold)
roi_color_undist_Img = np.copy(undist_Img)
img_shape_y = shape[0]
ima_shape_x = shape[1]
y_line = img_shape_y - 260
x_left = ima_shape_x - 715
x_right = ima_shape_x - 560
bottom_left = 90
bottom_right = ima_shape_x - 90
vertices, roi_marked_img = fnc.polygon_roi_color(roi_color_undist_Img,bottom_left, bottom_right,y_line, x_left, x_right, True)
Now the ROI is defined, so we can warp the ROI as follows.
warped = line.perspective_transform(undist_Img, vertices)
#plt.imshow(warped)
images=[undist_Img,warped]
titels=['Original','warped']
fnc.plot_n(images,titels,'gray')
# Warpe the Image
# for Binary Image
roi_color_undist_Img_bin = np.copy(undist_Img)
sobel_thresh = (30,220)
mag_thresh = (30, 220)
dir_thresh = (0.6, 1.1)
bin_thresh = (180, 255)
rgb_thresh = (200, 255)
hls_L_thresh = (90, 255)
hls_H_thresh = (15, 100)
hls_thresh = hls_L_thresh + hls_H_thresh
thresh = [sobel_thresh, mag_thresh, dir_thresh, bin_thresh, \
rgb_thresh, hls_thresh]
ksize = 9
binary_best_combi = line.pipline_combination(roi_color_undist_Img_bin, ksize, thresh, False)
vertices = fnc.polygon_roi_binary(binary_best_combi,bottom_left, bottom_right,y_line, x_left, x_right)
warped_bin = line.perspective_transform(binary_best_combi, vertices)
plt.imshow(undist_Img)
images=[binary_best_combi,warped_bin]
titels=['Best Binary Combination','Warped Binary Image']
fnc.plot_n(images,titels,'gray')
With the histogram it is possible to find the lane lines in the image. For further analysis we get now a start point of the lanes in the image.
# Histogram
histogram = line.hist(warped_bin, True)
We can use the two highest peaks from our histogram as a starting point for determining where the lane lines are, and then use sliding windows moving upward in the image (further along the road) to determine where the lane lines go.
Next step is to set a few hyperparameters related to our sliding windows, and set them up to iterate across the binary activations in the image.
Now that we have found all our pixels belonging to each line through the sliding window method, it's time to fit a polynomial to the line like:
$ f(y)=Ay^2+By+C $
The used function and the implementation is in FindingLane.py.
%%html
<img src="Bilder Notebook/Sliding Window.png", width=400>
%%html
<img src="Bilder Notebook/R_Kurve.jpg", width=400>
# HYPERPARAMETERS
# Choose the number of sliding windows
nwindows = 9
# Set the width of the windows +/- margin
margin = 100
# Set minimum number of pixels found to recenter window
minpix = 50
find_pixel_out_img = line.find_pixels(warped_bin, nwindows, margin, minpix, True)
ployfit_out_img = line.fit_polinominal(find_pixel_out_img,find_pixel_out_img, True)
Next step is to calculate the Radius of the Curvature and the Out of Center value. The Radius can be calculated like this:
%%html
<img src="Bilder Notebook/R_Formel_aufl.png", width=200>
ooc = line.calc_curvature_ooc(warped_bin)
print('left_curv: ' + str(line.left_curverad) + 'm'
'right_curv: ' + str(line.right_curverad) + 'm')
print('OOC: ' + str(ooc))
undist_Img_copy = np.copy(undist_Img)
# Draw lane on Original Image
colerd_lane_img = line.draw_lane_on_image(undist_Img_copy,warped_bin)
img_text = "Radius of curvature: " + str(round((line.left_curverad + line.right_curverad)/2,2)) + ' (m)'
if ooc<+ 0:
img_text2 = "Vehicle is: " + str(round(np.abs(ooc),2)) + ' (m) left of center'
else:
img_text2 = "Vehicle is: " + str(round(np.abs(ooc),2)) + ' (m) right of center'
cv2.putText(colerd_lane_img, img_text, (40, 70), cv2.FONT_HERSHEY_DUPLEX, 1.5, (255, 255, 255), 2)
cv2.putText(colerd_lane_img, img_text2, (40, 140), cv2.FONT_HERSHEY_DUPLEX, 1.5, (255, 255, 255), 2)
plt.imshow(colerd_lane_img)
plt.show()
This is a summery of the used function in one pipline.
def pipeline(undist_Img, plot_flag):
#####Sobel Threshold
sobel_thresh = (30,220)
mag_thresh = (30, 220)
dir_thresh = (0.6, 1.1)
bin_thresh = (180, 255)
rgb_thresh = (200, 255)
hls_L_thresh = (90, 255)
hls_H_thresh = (15, 100)
hls_thresh = hls_L_thresh + hls_H_thresh
thresh = [sobel_thresh, mag_thresh, dir_thresh, bin_thresh, \
rgb_thresh, hls_thresh]
ksize = 9
binary_best_combi = line.pipline_combination(undist_Img, ksize, thresh, plot_flag)
# ROI - Four Side Polygone -> polygon_roi(img, imshape, y_line, x_left, x_right, rgb_threshold)
img_shape_y = binary_best_combi.shape[0]
ima_shape_x = binary_best_combi.shape[1]
y_line = img_shape_y - 275
x_left = ima_shape_x - 675
x_right = ima_shape_x - 610
bottom_left = 180
bottom_right = ima_shape_x - 150
# for Binary Image
vertices = fnc.polygon_roi_binary(binary_best_combi,bottom_left, bottom_right,y_line, x_left, x_right)
warped_bin = line.perspective_transform(binary_best_combi, vertices)
# Histogram
histogram = line.hist(warped_bin, plot_flag)
# HYPERPARAMETERS
# Choose the number of sliding windows
nwindows = 9
# Set the width of the windows +/- margin
margin = 100
# Set minimum number of pixels found to recenter window
minpix = 50
out = line.find_pixels(warped_bin, nwindows, margin, minpix, plot_flag)
ployfit_out_img = line.fit_polinominal(warped_bin, out, plot_flag)
# Calc Curvature
ooc = line.calc_curvature_ooc(warped_bin)
undist_Img_copy = np.copy(undist_Img)
# Draw lane on Original Image
colerd_lane_img = line.draw_lane_on_image(undist_Img_copy,warped_bin)
colerd_lane_img.astype(np.uint8)
img_text = "Radius of curvature: " + str(round((line.left_curverad + line.right_curverad)/2)) + ' (m)'
if ooc<+ 0:
img_text2 = "Vehicle is: " + str(round(np.abs(ooc),2)) + ' (m) left of center'
else:
img_text2 = "Vehicle is: " + str(round(np.abs(ooc),2)) + ' (m) right of center'
cv2.putText(colerd_lane_img, img_text, (40, 70), cv2.FONT_HERSHEY_DUPLEX, 1.5, (255, 255, 255), 2)
cv2.putText(colerd_lane_img, img_text2, (40, 140), cv2.FONT_HERSHEY_DUPLEX, 1.5, (255, 255, 255), 2)
return colerd_lane_img
import pickle
import glob
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import math
import function as fnc
import FindingLane as lane
import scipy.misc
%matplotlib inline
# Read in the saved camera matrix and distortion coefficients
# These are the arrays you calculated using cv2.calibrateCamera()
dist_pickle = pickle.load( open("Camera_Calib_Values/wide_dist_pickle9Corner_class.p", "rb" ) )
mtx = dist_pickle["mtx"]
dist = dist_pickle["dist"]
images = glob.glob('test_images/TestPipline/*.jpg')
for idx, fname in enumerate(images):
color_img, shape = fnc.readIMG(fname)
line = lane.Lines()
line.set_cam_calib_param(mtx,dist)
undist_Img = line.undistort(color_img)
outputIMG = pipeline(undist_Img, None)
plt.imshow(outputIMG)
plt.show()
# Save Image
#write_name = 'output_images/Colored_Lane_Measured/Output_Img_Colored' + str(idx)+'.jpg'
#scipy.misc.imsave(write_name, outputIMG)
import pickle
import glob
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import math
import function as fnc
import FindingLane as lane
import scipy.misc
%matplotlib inline
def image_pipeline(img):
# Read in the saved camera matrix and distortion coefficients
# These are the arrays you calculated using cv2.calibrateCamera()
dist_pickle = pickle.load( open("Camera_Calib_Values/wide_dist_pickle9Corner_class.p", "rb" ) )
mtx = dist_pickle["mtx"]
dist = dist_pickle["dist"]
#line = lane.Lines()
line.set_cam_calib_param(mtx,dist)
undist_Img = line.undistort(img)
outputIMG = pipeline(undist_Img, None)
return outputIMG
from moviepy.editor import VideoFileClip
from IPython.display import HTML
#my_clip.write_gif('test.gif', fps=12)
line = lane.Lines()
video_output1 = 'project_video_output4.mp4'
video_input1 = VideoFileClip('project_video.mp4')#.subclip(22,26)
processed_video = video_input1.fl_image(image_pipeline)
%time processed_video.write_videofile(video_output1, audio=False)